package bundle2name

import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.Spider
import us.codecraft.webmagic.processor.PageProcessor

val baiduUrls = arrayOf("http://shouji.baidu.com/software/503","http://shouji.baidu.com/game/401/")
fun main(args: Array<String>) {
    val sp = Spider.create(BaiduPP())
            .addUrl(*baiduUrls)
            .addPipeline(FilePipeline())
            .thread(16)
    sp.run()
}

class BaiduPP : PageProcessor{
    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3).setTimeOut(300_000).setRetrySleepTime(10_000)

    }

    override fun process(page: Page) {
        println(page.url)
        val apps = page.html.xpath("//div[@class='app-bd']/ul/li")
        apps?.let {
            val ps = mutableListOf<BundleName>()
            it.nodes().forEach {
                val name = it.xpath("//p[@class='name']/text()").get()
                val bundle = it.xpath("//p[@class='down-btn']/span/@data_package").get()
                ps.add(BundleName(bundle,name))
            }
            page.putField("ps",ps)

            val tags = page.html.xpath("//ul[@class='yui3-u type-name']/li/a").links().all()
            tags?.let {
                page.addTargetRequests(it.filterNot { page.url.get().contains(it) })

            }

            val pager = page.html.xpath("//div[@class='pager']/ul/li").links().all()
            pager?.let {
                page.addTargetRequests(if (it.size > 1) it.subList(1,it.size - 1) else it)
            }
        }
    }

}